const request=require("superagent");
const https = require('https');
const cheerio = require('cheerio');
const iconv = require('iconv-lite');
const util = require('util');
const charset = require("superagent-charset")(request);
const fs=require('fs');
var mysql      = require('mysql');
const assert = require('assert');
const fsp = require('fs').promises;
const { Document, Packer, Paragraph, TextRun } = require('docx');
// var connection = mysql.createConnection({
//   host     : '39.103.57.165',
//   user     : 'sytc_test',
//   password : 'f2nTHDebtAkJnYRk',
//   database : 'sytc_test'
// });
connection.query = util.promisify(connection.query);

//插入数据库
async function asySql(addSql,addSqlParams){
    return await new Promise(resolve => {
        connection.query(addSql,addSqlParams,function (err, result) {
            if(err){
             console.log('[INSERT ERROR] - ',err.message);
             return;
            }else{
                console.log('INSERT ID:',result.insertId);
                return resolve(result.insertId);
            }        
            
        });
    })
        
}



async function getResourceTree(page) {
    var resource = await page._client.send('Page.getResourceTree');
    return resource.frameTree;
}

async function getResourceContent(page, url) {
    const { content, base64Encoded } = await page._client.send(
        'Page.getResourceContent',
        { frameId: String(page.mainFrame()._id), url },
    );
    assert.equal(base64Encoded, true);
    return content;
};
const Agent=[
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",
    "Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn; MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "JUC (Linux; U; 2.3.7; zh-cn; MB200; 320*480) UCWEB7.9.3.103/139/999",
    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:7.0a1) Gecko/20110623 Firefox/7.0a1 Fennec/7.0a1",
    "Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10",
    "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 3_0 like Mac OS X; en-us) AppleWebKit/420.1 (KHTML, like Gecko) Version/3.0 Mobile/1A542a Safari/419.3",
    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_0 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8A293 Safari/6531.22.7",
    "Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10",
    "Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 Mobile Safari/534.1+",
    "Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.0; U; en-US) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/233.70 Safari/534.6 TouchPad/1.0",
    "Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18124",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)",
    "Mozilla/5.0 (Linux; U; Android 8.1.0; zh-cn; BLA-AL00 Build/HUAWEIBLA-AL00) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.132 MQQBrowser/8.9 Mobile Safari/537.36",
];
const timeout={
    response:50000,
    deadline:600000,
}
Date.prototype.format = function (fmt) {
    var o = {
        "M+": this.getMonth() + 1, //月份
        "d+": this.getDate(), //日
        "h+": this.getHours(), //小时
        "m+": this.getMinutes(), //分
        "s+": this.getSeconds(), //秒
        "q+": Math.floor((this.getMonth() + 3) / 3), //季度
        "S": this.getMilliseconds() //毫秒
    };
    if (/(y+)/.test(fmt)) fmt = fmt.replace(RegExp.$1, (this.getFullYear() + "").substr(4 - RegExp.$1.length));
    for (var k in o)
        if (new RegExp("(" + k + ")").test(fmt)) fmt = fmt.replace(RegExp.$1, (RegExp.$1.length == 1) ? (o[k]) : (("00" + o[k]).substr(("" + o[k]).length)));
    return fmt;
}
function delay(t){
    return new Promise(resolve => setTimeout(resolve, t))
}

function rnd(n, m){
    var random = Math.floor(Math.random()*(m-n+1)+n);
    return random;
}
function getRndIP(){
    var ip=[];
    for(var i=0;i<4;i++){
    ip.push(rnd(10,250));
    }
    return ip.join(".");
}

const agent_c = new https.Agent({
  rejectUnauthorized: false
});

// 配置全局错误捕获
request.Request.prototype._callback = function(err, res) {
  if (err) {
    this.reject(err); // 强制将错误传递到 Promise 的 reject
  } else {
    this.resolve(res);
  }
};
// process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
//封装请求 实现失败重试
async function requestWithRetry(url, retryCount = 3, timeout = 5000,option) {
  return new Promise((resolve, reject) => {
    let numRetries = 0;
    const makeRequest = () => {

         request
            .get(url)
            .set(option).on('error', (err) => {
                  console.error('捕获异常b:', err.message);
                    if (numRetries < retryCount) {
                        numRetries += 1;
                        console.log(`Request timed out. Retrying... (Attempt ${numRetries})`);
                        makeRequest(); // 重试请求
                    }
                })
            .agent(agent_c)
            .timeout(timeout) // 设置超时时间
            .end((err, res) => {
              if (err) {
                console.error('HTTP错误:', err.message);
                numRetries += 1;
                console.log(`Request timed out. Retrying... (Attempt ${numRetries})`);
                makeRequest(); // 重试请求
                }else{
                    resolve(res);
                }
            });
            // .then(response => {
            //   resolve(response);
            // })
            // .catch(error => {
            //     console.log('捕获异常a')
                
            //   if (error.code === 'ECONNABORTED' && numRetries < retryCount) {
            //     numRetries += 1;
            //     console.log(`Request timed out. Retrying... (Attempt ${numRetries})`);
            //     makeRequest(); // 重试请求
            //   } else {
            //     reject(error);
            //   }
            // });
            process.on('unhandledRejection', (err) => {
              console.error('未处理的 Promise 错误:', err.message);
              makeRequest();
            });
            process.on('uncaughtException', (err) => {
              console.error('未捕获的异常:', err.message);
               makeRequest();
            });
      
    };
 
    makeRequest();
  });
}

async function getUnit(){
    let serverType = 'personal';//个人
    // let serverType = 'corporation';//公司
    let sql=`select * from m_area where id = 108`;
    let result=await connection.query(sql);
    for (var i = 0; i < result.length; i++) {
            var ip=getRndIP();
            var agent=Agent[rnd(0,Agent.length-1)];
            agent=Agent[0];
            var option={
                "Host":"zwfw.nmg.gov.cn",
                "User-Agent":agent,
                "Referer":'https://zwfw.nmg.gov.cn/',
                "X-Forwarded-For":ip,
            }
            let url=`https://zwfw.nmg.gov.cn/handle_affairs?regionCode=${result[i]['code']}&workType=${serverType}&record=istrue`;
            // let url=`https://zwfw.dl.gov.cn/dlPortal/getCategoryByItem?areacode=${result[i]['code']}&serverType=${serverType}`;
            console.log(result[i]['id'],result[i]['name'],url);
            let res = await requestWithRetry(url, 3, 5000,option).then(response => {
                // console.log('Response received:', response);
                return response;
              }).catch(error => {
                console.error('Error:', error);
              });
            let body=res.text;
            var $ = cheerio.load(body);
            //部门
            $('.VueGeRenAnBuMens:eq(0) li:gt(0)').each(async function(){
                let code = $(this).attr('id');
                let name = $(this).find('a').text();
                console.log(code,name)
                let data = {
                    'area_id' : result[i]['id'],
                    'code' : code,
                    'name' : name
                };
                let checkResult = await connection.query(`select * from m_unit where code = '${code}'`);
                if(checkResult.length>0){
                    console.error('数据已存在', name);
                }else{
                    await asySql("INSERT INTO m_unit SET  ?",{
                        'area_id' : result[i]['id'],
                        'code' : code,
                        'name' : name
                    }).then(insertId => {
                        return insertId;
                    }).catch(error => {
                      console.error(error); // 异步操作失败，输出错误信息
                    });
                }

            })
            
            await delay(1000);
            
        }
        
          
process.exit(0);  
}


async function getList(){
    let sql=`select * from m_area where id = 108`;
    let result=await connection.query(sql);
    for (var i = 0; i < result.length; i++) {
        //区域id
        let area_id = result[i]['id'];
        //区域编码
        let area_code = result[i]['code'];
        //区域名称
        let area_name = result[i]['name'];


        //循环部门爬取 方便目录插入部门
        let unitResult = await connection.query(`select * from m_unit where area_id = ${result[i]['id']}`);
        for (var u = 0; u < unitResult.length; u++) {
            //部门id
            let unit_id = unitResult[u]['id'];
            //部门编码
            let OrgCode = unitResult[u]['code'];
            //部门名称
            let unit_name = unitResult[u]['name'];
            //创建文件夹
            let filepath = `./事项数据/${area_name}/${unit_name}`;
            fs.mkdirSync(filepath, { recursive: true });

            var ip=getRndIP();
            var agent=Agent[rnd(0,Agent.length-1)];
            agent=Agent[0];
            var option={
                "Host":"zwfw.nmg.gov.cn",
                "User-Agent":agent,
                "Referer":'https://zwfw.nmg.gov.cn/',
                "X-Forwarded-For":ip,
            }
            let url=`https://zwfw.nmg.gov.cn/handle_affairs_serch_new?ItemType=&IsIntoHall=&SubjectClassify=&CurrentPage=1&PageSize=90000&OrgCode=${OrgCode}&RegionCode=${result[i]['code']}&ItemNameLike=&IsOnline=`;
            console.log(result[i]['id'],result[i]['name'],unitResult[u]['name'],url);
            //拿到该区域该部门下的事项列表 
            let res = await requestWithRetry(url, 3, 5000,option).then(response => {
                // console.log('Response received:', response);
                return response;
              }).catch(error => {
                console.error('Error:', error);
              });
            let resData = JSON.parse(res.text);
            resData = resData.itemJsonDataList;

            //循环事项列表
            for (var j = 0; j < resData.length; j++) {
                //判断ItemList是否为空
                let itemList = resData[j]['ItemList'];
                if(itemList==null){
                    for(var list1_i = 0; list1_i < resData[j]['SubFolder'].length;list1_i++){
                        if(resData[j]['SubFolder'][list1_i]['ItemList']==null){

                            for(var list2_i = 0; list2_i < resData[j]['SubFolder'][list1_i]['SubFolder'].length;list2_i++){
                                //事项处理
                                itemList = resData[j]['SubFolder'][list1_i]['SubFolder'][list2_i]['ItemList']
                                await saveMatter(itemList,filepath,area_id,unit_id,area_name,unit_name);
                            }
                        }else{
                           //事项处理
                            itemList = resData[j]['SubFolder'][list1_i]['ItemList']
                            await saveMatter(itemList,filepath,area_id,unit_id,area_name,unit_name); 
                        }
                        
                    }
                }else{
                    await saveMatter(itemList,filepath,area_id,unit_id,area_name,unit_name);
                }

            }



        }

        
        
            
    }
        
          
process.exit(0);  
}

//保存事项
async function saveMatter(itemList,filepath,area_id,unit_id,area_name,unit_name) {
    if(itemList!=null){
        for(var k = 0; k < itemList.length; k++) {
            let data = {
                'gw_link':`https://zwfw.nmg.gov.cn/handle_affairs_detail?item_detail_id=${itemList[k]['ItemID']}&regionCode=150000&highlight=&record=istrue`,
            };
            let sql=`select * from psm_business_unit WHERE title ='${itemList[k]['ItemName']}' AND did=162 AND gw_link IS NOT NULL`;
            let result=await connection.query(sql);
            if(result.length>0){
             console.log(`add ${itemList[k]['ItemName']} error`);
            }else{
                let addSql=`update psm_business_unit SET ? where title ='${itemList[k]['ItemName']}' AND did=162`;
                let addSqlParams = data;
                return await new Promise(resolve => {
                    connection.query(addSql,addSqlParams,function (err, result) {
                        if(err){
                         console.log('[INSERT ERROR] - ',err.message);
                         return;
                        }else{
                            console.log('UPDATA ID:',itemList[k]['ItemName']);
                            return resolve(result.insertId);
                        }        
                        
                    });
                })
                
            } 

        }
    }
    
}

//获取部门列表
// getUnit();

//获取事项列表
getList();

